package test.webmagic;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.selenium.SeleniumDownloader;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.RedisScheduler;

/**
 * 花瓣网抽取器。<br>
 * 使用Selenium做页面动态渲染。<br>
 */
public class HuabanProcessor implements PageProcessor {
    
    private Site site;
    
    @Override
    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links().regex("http://huaban\\.com/.*").all());
        if (page.getUrl().toString().contains("favorite")) {
            System.out.println("matchImg: " + page.getHtml().xpath("//a[@class='img x layer-view loaded']/img/@src").toString());
        } else {
            page.getResultItems().setSkip(true);
        }
    }
    
    @Override
    public Site getSite() {
        if (site == null) {
            site = Site.me().setDomain("huaban.com").setSleepTime(1000);
        }
        return site;
    }
  //*[@id="waterfall"]/div[10]/a/img
    public static void main(String[] args) {
        System.setProperty("selenuim_config",
            "C:\\Atos\\Mike\\GitRepos\\Mitdy-SportGame\\Mitdy-SportGame\\src\\main\\resources\\config.ini");
        Spider.create(new HuabanProcessor())
            .thread(5)
            .addUrl("http://huaban.com/")
            .scheduler(new QueueScheduler())
            .pipeline(new FilePipeline("C:\\迅雷下载"))
            .downloader(new SeleniumDownloader("C:\\Atos\\Mike\\ChromeDriver\\chromedriver.exe"))
            .run();
    }
}